#!/usr/bin/env python
# --coding:utf-8--
"""
@File    : Crawl.py
@Time    : 2021/8/2 18:59
@Author  : cpl
"""
import time
from npm_utils import npm_VersionUtils
import GlobalVar
from DatabaseOP import App

class Crawl:
    def __init__(self):
        # self.root_pro = 'no-one-left-behind'
        # self.root_version = '2018.2.10'
        # self.root_pro = 'react-native-ok-sdk'
        # self.root_version = '0.0.1'
        # self.root_pro = 'mongodb'
        # self.root_version = '4.0.1'

        self.url = "neo4j://localhost:7687"
        # driver = neo4j.GraphDatabase.driver(uri, auth=("neo4j", "sdpneo4j"))
        self.app = App(self.url, "neo4j", "sdpneo4j")

        GlobalVar._init()
        self.Verutils = npm_VersionUtils()
        self.miss = 0
        self.noverison = 0
        self.name = ''
        self.pkg_info = ''
        self.version_num = 0

    def close(self):
        self.app.close()


    def set_pkg(self, name):
        self.name = name
        self.Verutils.set_name_version(name)
        self.pkg_info = self.Verutils.load_pkgjson()
        if 'error' in self.pkg_info:
            print('error: can not load pkg info: %s' % (name))
            self.version_num = 0
            return False
        self.version_num = len(self.pkg_info['versions']) if 'versions' in self.pkg_info.keys() else 1
        return True


    def crawl_allversion_by_batch(self, name, batch_size):
        start_time = time.time()
        self.set_pkg(name)
        
        self.last_ordernum = GlobalVar.get_ordernum()
        self.max_insert_speed = 0
        self.max_batch_order = 0
        self.max_batch_time = 0

        x = 0
        while x <= self.version_num:
            GlobalVar.add_queue(name, 'INIT')
            with self.app.driver.session() as session:
                with session.begin_transaction() as tx:
                    self.crawl_batch_version(tx, x, x + batch_size + 1)
                    tx.commit()
            x += batch_size + 1

        self.app.close()
        end_time = time.time()
        print('\nFinish pkg %s all %d versions:\n'%(name, self.version_num), 'batch size: %d\n'%(batch_size), 'Total order num: %d\n'%(GlobalVar.get_ordernum()),'Time cosuming: %f s\n'%(end_time - start_time),'Max insert speed: %f order/s, batch order: %d, babtch time: %f s\n'%(self.max_insert_speed, self.max_batch_order, self.max_batch_time),
              'Miss pkgs: %d\n'%(self.miss), 'Bad version constraint pkgs: %d\n'%(self.noverison)
              )


    def crawl_batch_version(self, tx, st, ed):
        if ed > self.version_num:
            ed = self.version_num + 1
        start_time = time.time()
        self.app.load_allversion(tx, self.pkg_info, self.name, st, ed)
        num = GlobalVar.get_qsize()
        while num > 0:
            for i in range(num):
                pro_name, pro_version = GlobalVar.pop_queue()
                self.Verutils.set_name_version(pro_name)
                rlt = self.Verutils.load_pkgjson()
                if 'error' in rlt:
                    print('error: can not load pkg info: %s'%(pro_name))
                    continue




                self.app.load_pkg(tx, rlt, pro_name, pro_version)

            num = GlobalVar.get_qsize()

        print('---------------batch %d-%d ---------------'%(st, ed))
        batch_order = GlobalVar.get_ordernum() - self.last_ordernum
        self.last_ordernum = GlobalVar.get_ordernum()
        print('insert order num: %d'%(batch_order))
        batch_time = time.time() - start_time
        print('time consuming: %f s'%batch_time)
        batch_speed = batch_order / batch_time
        print('batch insert speed: %f order/s'%(batch_speed))
        if batch_speed > self.max_insert_speed:
            self.max_insert_speed = batch_speed  
            self.max_batch_order = batch_order
            self.max_batch_time = batch_time

        if GlobalVar.count_miss() > 0:
            self.miss += GlobalVar.count_miss()
            print("miss %d packages due to 403/404!"%(GlobalVar.count_miss()))
            GlobalVar.write_miss()
            GlobalVar.clear_miss()

        if GlobalVar.count_noversion() > 0:
            self.noverison += GlobalVar.count_noversion()
            print("%d packages' version constraints cant be resolved" % (GlobalVar.count_noversion()))
            GlobalVar.write_noversion()
            GlobalVar.clear_noversion()


    def crawl_batch(self, name, version, batch_size):
        if name == '':
            print("name is illegal！")
            return None
        elif version != '':
            GlobalVar.add_queue(name, version)
            GlobalVar.add_seen(name + '@' + version)

        num = GlobalVar.get_qsize()
        start_time = time.time()
        session = self.app.driver.session()
        tx = session.begin_transaction()
        order_num = 0
        if(name == ''):
            name = 'pkg.list'
            self.version_num=1
        else:
            self.set_pkg(name)
            if GlobalVar.not_seeall(name):
                self.app.load_allversion(tx, self.pkg_info, name, 0, 0)
        while num > 0:
            for i in range(num):
                pro_name, pro_version = GlobalVar.pop_queue()
                self.Verutils.set_name_version(pro_name)
                rlt = self.Verutils.load_pkgjson()
                if 'error' in rlt:
                    print('error: can not load pkg info: %s' % (pro_name))
                    continue
                #if GlobalVar.not_seeall(pro_name):
                #    self.app.load_allversion(tx, rlt, pro_name)

                self.app.load_pkg(tx, rlt, pro_name, pro_version)
                if GlobalVar.get_ordernum() >= batch_size:
                    tx.commit()
                    order_num += GlobalVar.get_ordernum()
                    GlobalVar.clear_ordernum()
                    session.close()
                    session = self.app.driver.session()
                    tx = session.begin_transaction()

            num = GlobalVar.get_qsize()
        tx.commit()
        print('---------------finish %s-all %d version ---------------' % (name, self.version_num))
        print('insert batch size: %d'%(batch_size))
        total_time = time.time() - start_time
        print('total orders: %d'%(order_num))
        print('time consuming: %f s' % (total_time))
        print('average inserting speed: %f order/s' % (order_num / total_time))
        if GlobalVar.count_miss() > 0:
            self.miss += GlobalVar.count_miss()
            ## record.write("miss %d packages due to 403/404!\n" % (GlobalVar.count_miss()))
            print("miss %d packages due to 403/404!" % (GlobalVar.count_miss()))
            GlobalVar.write_miss(name, self.version_num)
            GlobalVar.clear_miss()

        if GlobalVar.count_noversion() > 0:
            self.noverison += GlobalVar.count_noversion()
            ## record.write("%d packages' version constraints cant be resolved\n" % (GlobalVar.count_noversion()))
            print("%d packages' version constraints cant be resolved" % (GlobalVar.count_noversion()))
            GlobalVar.write_noversion(name, self.version_num)
            GlobalVar.clear_noversion()


    def crawl_list_batch(self, batch_size):
        pkgs = GlobalVar.get_pkg_list()
        # pkg_num = len(pkgs)
        st = time.time()
        session = self.app.driver.session()
        tx = session.begin_transaction()
        allv_pkg_index = 0
        for cur_pkg in pkgs:
            if not GlobalVar.not_seeall(cur_pkg):
                print("%s is already exist！"%(cur_pkg))
                continue

            APIresponse = self.set_pkg(cur_pkg)
            if not APIresponse:
                print('error: can not load pkg info: %s' % (cur_pkg))
                continue

            allv_pkg_index += 1
            GlobalVar.add_allvnum()
            self.app.load_allversion(tx, self.pkg_info, cur_pkg, 0, 0)
            num = GlobalVar.get_qsize()
            start_time = time.time()
            order_num = 0
            while num > 0:
                for i in range(num):
                    pro_name, pro_version = GlobalVar.pop_queue()
                    self.Verutils.set_name_version(pro_name)
                    rlt = self.Verutils.load_pkgjson()
                    if 'error' in rlt:
                        print('error: can not load pkg info: %s' % (pro_name))
                        continue

                    self.app.load_pkg(tx, rlt, pro_name, pro_version)

                    if GlobalVar.get_ordernum() >= batch_size:
                        tx.commit()
                        order_num += GlobalVar.get_ordernum()
                        GlobalVar.clear_ordernum()
                        session.close()
                        session = self.app.driver.session()
                        tx = session.begin_transaction()

                num = GlobalVar.get_qsize()



            # 打开一个文件
            record = open("summary.txt", "a")

            record.write('\n---------------finish NO.%d %s-all %d version ---------------\n' % (allv_pkg_index, cur_pkg, self.version_num))
            print('\n---------------finish NO.%d %s-all %d version ---------------' % (allv_pkg_index, cur_pkg, self.version_num))

            ## record.write('insert batch size: %d\n' % (batch_size))
            print('insert batch size: %d' % (batch_size))
            total_time = time.time() - start_time
            record.write('total orders: %d\n' % (order_num))
            record.write('time consuming: %f s\n' % (total_time))
            print('total orders: %d' % (order_num))
            print('time consuming: %f s' % (total_time))
            print('average inserting speed: %f order/s' % (order_num / total_time))
            if GlobalVar.count_miss() > 0:
                self.miss += GlobalVar.count_miss()
                record.write("miss %d packages due to 403/404!\n" % (GlobalVar.count_miss()))
                print("miss %d packages due to 403/404!" % (GlobalVar.count_miss()))
                GlobalVar.write_miss(cur_pkg, self.version_num)
                GlobalVar.clear_miss()

            if GlobalVar.count_noversion() > 0:
                self.noverison += GlobalVar.count_noversion()
                record.write("%d packages' version constraints cant be resolved\n" % (GlobalVar.count_noversion()))
                print("%d packages' version constraints cant be resolved" % (GlobalVar.count_noversion()))
                GlobalVar.write_noversion(cur_pkg, self.version_num)
                GlobalVar.clear_noversion()

            print('-------------------------------------------------------')
            record.write('-------------------------------------------------------\n')
            record.close()
        tx.commit()
        self.close()
        ed = time.time()
        pkg_num = GlobalVar.get_allvnum()
        print("FINISH %d PKGS ALL VERSIONS IN pkgs list(%d)" %(pkg_num, len(pkgs)))
        print('TIME CONSUMING: %f s' % (ed -st))
        record = open("summary.txt", "a")
        record.write("FINISH %d PKGS ALL VERSIONS IN pkgs list(%d)\n" % (pkg_num, len(pkgs)))
        record.write('TIME CONSUMING: %f s\n' % (ed - st))
        record.close()

    def crawl_vul_pkg_batch(self, tx, name, batch_size):
        if name == '':
            print("name is illegal！")
            return None
        num = GlobalVar.get_qsize()
        self.set_pkg(name)
        if GlobalVar.not_seeall(name):
            self.app.load_allversion(tx, self.pkg_info, name, 0, 0)

        while num > 0:
            for i in range(num):
                pro_name, pro_version = GlobalVar.pop_queue()
                self.Verutils.set_name_version(pro_name)
                rlt = self.Verutils.load_pkgjson()
                if 'error' in rlt:
                    print('error: can not load pkg info: %s' % (pro_name))
                    continue

                self.app.load_pkg(tx, rlt, pro_name, pro_version)
            num = GlobalVar.get_qsize()

        print('---------------finish %s-all %d version ---------------' % (name, self.version_num))

        # 打开一个文件
        record = open("summary.txt", "a")

        record.write('\n---------------finish %s-all %d version ---------------\n' % (name, self.version_num))

        if GlobalVar.count_miss() > 0:
            self.miss += GlobalVar.count_miss()
            record.write("miss %d packages due to 403/404!\n" % (GlobalVar.count_miss()))
            print("miss %d packages due to 403/404!" % (GlobalVar.count_miss()))
            GlobalVar.write_miss(name, self.version_num)
            GlobalVar.clear_miss()

        if GlobalVar.count_noversion() > 0:
            self.noverison += GlobalVar.count_noversion()
            record.write("%d packages' version constraints cant be resolved\n" % (GlobalVar.count_noversion()))
            print("%d packages' version constraints cant be resolved" % (GlobalVar.count_noversion()))
            GlobalVar.write_noversion(name, self.version_num)
            GlobalVar.clear_noversion()

        record.write('-------------------------------------------------------\n')
        record.close()